#!/bin/bash
#set  -x
if [ $# -eq 1 ]; then
    day="$1"
else
    day=`date +%Y%m%d --date '-1 day'`
fi

downLoadFiles(){
	reqUrl='http://dsap2.web.58dns.org/dsap-agent/?targetserver=db-cdh-15-229&targetport=50075&targetpage=browseDirectory&targetprefix=jsp&dir='$1'&namenodeInfoPort=50070&nnaddr=10.9.14.196:9000'	
	nameList=`wget "$reqUrl" -O- 2>/dev/null|grep -Po '(?<=:9000">)[^/]*?(?=</a></B><br></td>)'|sed '/^_/d'`
	getParams=$(for name in $nameList
	do 
		downFileFullName=$1/$name
		tmpUrl="http://10.9.15.19:50075/streamFile$downFileFullName?nnaddr=10.9.14.196:9000"
		echo $tmpUrl
	done)
	wget -O $2 $getParams
	
#python 
# src='http://10.9.15.19:50075/streamFile/dsap/resultdata/ec_data/EC_Post_Detail_Job/20141028/part-r-00000?nnaddr=10.9.14.196:9000'
# target="part-r-00000.txt"
# urllib.urlretrieve(src,target)   #python3中是urllib.request.urlretrieve()
################################
} 

user_post_path="/data/logs/wuba/user/post/file/user_post_${day}"
user_attr_path="/data/logs/wuba/user/post/file/user_attr_${day}"
downLoadFiles "/dsap/middata/user_post_job/${day}/user_post" $user_post_path
downLoadFiles "/dsap/middata/user_post_job/${day}/user_attrs" $user_attr_path
#python /work/opt/fourthinfo/data/formatMysqlRow.py .  ${day} 1>logs/stdout_${day}.log 2>logs/stderr_${day}.log &
python user_post.py $day